{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# 中文分词"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['我要', '好好学习', '。', 'Pytorch', '框架', '零', '基础', '入门', '保姆', '级', '教程', '，', '快速', '搭建', '神经网络', '分类', '模型']\n",
      "['我', '要', '好好', '好好学', '好好学习', '好学', '学习', '。', 'Pytorch', '框架', '零基', '基础', '入门', '保姆', '级', '教程', '，', '快速', '搭建', '神经', '神经网', '神经网络', '网络', '分类', '模型']\n",
      "['我要', '好好', '好学', '学习', '好好学', '好好学习', '。', 'Pytorch', '框架', '零', '基础', '入门', '保姆', '级', '教程', '，', '快速', '搭建', '神经', '网络', '神经网', '神经网络', '分类', '模型']\n",
      "['好好', '好学', '学习', '好好学', '好好学习', ',', '天天', '向上', '天天向上']\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "\n",
    "seg_str = \"我要好好学习。Pytorch框架零基础入门保姆级教程，快速搭建神经网络分类模型\"\n",
    "str2 = \"好好学习,天天向上\"\n",
    "print(jieba.lcut(seg_str))  # 精简模式，返回一个列表类型的结果\n",
    "print(jieba.lcut(seg_str, cut_all=True))  # 全模式，使用 'cut_all=True' 指定\n",
    "print(jieba.lcut_for_search(seg_str))  # 搜索引擎模式\n",
    "print(jieba.lcut_for_search(str2))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 英文分词"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Robert', 'DeNiro', 'plays', 'the', 'most', 'unbelievably', 'intelligent', 'illiterate', 'of', 'all', 'time', 'This', 'movie', 'is', 'so', 'wasteful', 'of', 'talent', 'it', 'is', 'truly', 'disgusting', 'The', 'script', 'is', 'unbelievable', 'The', 'dialog', 'is', 'unbelievable', 'Jane', \"Fonda's\", 'character', 'is', 'a', 'caricature', 'of', 'herself', 'and', 'not', 'a', 'funny', 'one', 'The', 'movie', 'moves', 'at', 'a', \"snail's\", 'pace', 'is', 'photographed', 'in', 'an', 'ill', 'advised', 'manner', 'and', 'is', 'insufferably', 'preachy', 'It', 'also', 'plugs', 'in', 'every', 'cliche', 'in', 'the', 'book', 'Swoozie', 'Kurtz', 'is', 'excellent', 'in', 'a', 'supporting', 'role', 'but', 'so', 'what', 'Equally', 'annoying', 'is', 'this', 'new', 'IMDB', 'rule', 'of', 'requiring', 'ten', 'lines', 'for', 'every', 'review', 'When', 'a', 'movie', 'is', 'this', 'worthless', 'it', \"doesn't\", 'require', 'ten', 'lines', 'of', 'text', 'to', 'let', 'other', 'readers', 'know', 'that', 'it', 'is', 'a', 'waste', 'of', 'time', 'and', 'tape', 'Avoid', 'this', 'movie']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "#1. 定义tokenize的方法\n",
    "def tokenize(text):\n",
    "    # fileters = '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\n",
    "    fileters = ['!', '\"', '#', '$', '%', '&', '\\(', '\\)', '\\*', '\\+', ',', '-', '\\.', '/', ':', ';', '<', '=', '>',\n",
    "                '\\?', '@'\n",
    "        , '\\[', '\\\\', '\\]', '^', '_', '`', '\\{', '\\|', '\\}', '~', '\\t', '\\n', '\\x97', '\\x96', '”', '“', ]\n",
    "    text = re.sub(\"<.*?>\", \" \", text, flags=re.S)\n",
    "    text = re.sub(\"|\".join(fileters), \" \", text, flags=re.S)\n",
    "    return [i.strip() for i in text.split()]\n",
    "\n",
    "\n",
    "str = \"Robert DeNiro plays the most unbelievably intelligent illiterate of all time. This movie is so wasteful of talent, it is truly disgusting. The script is unbelievable. The dialog is unbelievable. Jane Fonda's character is a caricature of herself, and not a funny one. The movie moves at a snail's pace, is photographed in an ill-advised manner, and is insufferably preachy. It also plugs in every cliche in the book. Swoozie Kurtz is excellent in a supporting role, but so what?<br /><br />Equally annoying is this new IMDB rule of requiring ten lines for every review. When a movie is this worthless, it doesn't require ten lines of text to let other readers know that it is a waste of time and tape. Avoid this movie.\"\n",
    "print(tokenize((str)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}